# -- coding: utf-8 --
# -- coding: utf-8 --
import re
import json
import scrapy
from spidertools.utils.time_utils import get_current_date
from commonresources.spider_items.base_item import convert_dict
from commonresources.spider_items.anhui.items import AnHuiShengJiaoTongYunShuTingItem
from commonresources.spiders.basespider import BaseSpider
from pprint import pprint
import requests
import datetime
import time


class AnHuiShengJiaoTongYunShuTingSpider(BaseSpider):



    name = 'AnHuiShengJiaoTongYunShuTing'
    name_zh = "安徽省交通运输厅"
    province = "安徽"

    start_urls = ['http://jtt.ah.gov.cn/site/label/8888?IsAjax=1&dataType=html&_=0.44805718975466013&labelName=publicInfoList&siteId=6787991&pageSize=15&pageIndex=1&action=list&fuzzySearch=true&fromCode=title&keyWords=&sortType=0&isDate=true&dateFormat=yyyy-MM-dd&length=80&organId=21701&type=4&catIds=&cId=&result=%E6%9A%82%E6%97%A0%E7%9B%B8%E5%85%B3%E4%BF%A1%E6%81%AF&file=%2Fxxgk%2FpublicInfoList_newest2020&catId=6714761']


    def __init__(self, full_dose=True):
        super(AnHuiShengJiaoTongYunShuTingSpider, self).__init__(full_dose)
        self.convert_dict = convert_dict

    def parse(self, response):
        type_dict = {
                     '招标公告': 'http://jtt.ah.gov.cn/site/label/8888?IsAjax=1&dataType=html&_=0.032054491009384956&labelName=publicInfoList&siteId=6787991&pageSize=15&pageIndex=1&action=list&fuzzySearch=true&fromCode=title&keyWords=&sortType=0&isDate=true&dateFormat=yyyy-MM-dd&length=80&organId=21701&type=4&catIds=&cId=&result=%E6%9A%82%E6%97%A0%E7%9B%B8%E5%85%B3%E4%BF%A1%E6%81%AF&file=%2Fxxgk%2FpublicInfoList_newest2020&catId=6714761',
                     '中标公告': 'http://jtt.ah.gov.cn/site/label/8888?IsAjax=1&dataType=html&_=0.28415391274064095&labelName=publicInfoList&siteId=6787991&pageSize=15&pageIndex=1&action=list&fuzzySearch=true&fromCode=title&keyWords=&sortType=0&isDate=true&dateFormat=yyyy-MM-dd&length=80&organId=21701&type=4&catIds=&cId=&result=%E6%9A%82%E6%97%A0%E7%9B%B8%E5%85%B3%E4%BF%A1%E6%81%AF&file=%2Fxxgk%2FpublicInfoList_newest2020&catId=6714771',
                     }
        headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'Accept-Encoding': 'gzip, deflate',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
            'Cache-Control': 'max-age=0',
            'Connection': ' keep-alive',
            'Cookie': 'SHIROJSESSIONID=c5bd70c5-d911-4863-b335-2f96366cf1fc',
            'Host': 'jtt.ah.gov.cn',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36 Edg/90.0.818.66',
        }

        for key,value in type_dict.items():
            object_type = str(key)
            start_url = str(value)
            yield scrapy.Request(
                url=start_url,
                callback=self.first_parse,
                headers=headers,
                meta={
                    "need_break": False,
                    'type': object_type,
                    'page': 1,
                    'headers':headers
                },

            )


    def first_parse(self, response):
        if not response.meta['need_break']:
            object_type = response.meta['type']
            page = response.meta['page']
            if object_type =='招标公告' and page <300:
                page += 1
                new_url = 'http://jtt.ah.gov.cn/site/label/8888?IsAjax=1&dataType=html&_=0.032054491009384956&labelName=publicInfoList&siteId=6787991&pageSize=15&pageIndex='+str(page)+'&action=list&fuzzySearch=true&fromCode=title&keyWords=&sortType=0&isDate=true&dateFormat=yyyy-MM-dd&length=80&organId=21701&type=4&catIds=&cId=&result=%E6%9A%82%E6%97%A0%E7%9B%B8%E5%85%B3%E4%BF%A1%E6%81%AF&file=%2Fxxgk%2FpublicInfoList_newest2020&catId=6714761'
            elif object_type =='中标公告' and page <252:
                page += 1
                new_url = 'http://jtt.ah.gov.cn/site/label/8888?IsAjax=1&dataType=html&_=0.28415391274064095&labelName=publicInfoList&siteId=6787991&pageSize=15&pageIndex='+str(page)+'&action=list&fuzzySearch=true&fromCode=title&keyWords=&sortType=0&isDate=true&dateFormat=yyyy-MM-dd&length=80&organId=21701&type=4&catIds=&cId=&result=%E6%9A%82%E6%97%A0%E7%9B%B8%E5%85%B3%E4%BF%A1%E6%81%AF&file=%2Fxxgk%2FpublicInfoList_newest2020&catId=6714771'
            li_list = response.xpath('//ul[@class="clearfix xxgk_nav_list"]/li')
            for li in li_list:
                title = li.xpath('./a/text()').extract_first() #标题
                url = li.xpath('./a/@href').extract_first() #url
                release_time =li.xpath('./span/text()').extract_first() #发布时间
                item = {}
                item["announcement_type"] = response.meta['type']  # 公告类型
                item["announcement_title"] = title  # 标题
                item["release_time"] = release_time  # 发布时间
                item["origin_url"] = url  # 网站地址
                item["item"] = AnHuiShengJiaoTongYunShuTingItem()
                if not self.full_dose and item["release_time"] != get_current_date():
                    response.meta["need_break"] = True
                    break
                else:
                    time.sleep(1)
                    yield scrapy.Request(url=item["origin_url"],
                                         callback=self.parse_item_new,
                                         meta=item)

            yield scrapy.Request(
                url=new_url,
                callback=self.first_parse,
                headers=response.meta['headers'],
                meta={
                    "need_break": response.meta["need_break"],
                    'type': object_type,
                    'page': page,
                    'headers': response.meta['headers']
                },

            )



